from abc import ABCMeta, abstractmethod
import re
from xml.etree import ElementTree

import numpy as np

from vmaf.config import VmafConfig
from vmaf import ExternalProgramCaller
from vmaf.core.executor import Executor
from vmaf.core.niqe_train_test_model import NiqeTrainTestModel
from vmaf.core.result import Result
from vmaf.core.feature_assembler import FeatureAssembler
from vmaf.core.train_test_model import TrainTestModel, LibsvmNusvrTrainTestModel, \
    BootstrapLibsvmNusvrTrainTestModel
from vmaf.core.feature_extractor import SsimFeatureExtractor, MsSsimFeatureExtractor, \
    VmafFeatureExtractor
from vmaf.core.noref_feature_extractor import BrisqueNorefFeatureExtractor

__copyright__ = "Copyright 2016-2018, Netflix, Inc."
__license__ = "Apache, Version 2.0"


class QualityRunner(Executor):
    """
    QualityRunner takes in a list of assets, and run quality assessment on
    them, and return a list of corresponding results. A QualityRunner must
    specify a unique type and version combination (by the TYPE and VERSION
    attribute), so that the Result generated by it can be identified and
    stored by ResultStore class.

    There are two ways to create a derived class of QualityRunner:

    a) Call a command-line executable directly, very similar to what
    FeatureExtractor does. You must:
        1) Override TYPE and VERSION
        2) Override _generate_result(self, asset), which call a
        command-line executable and generate quality scores in a log file.
        3) Override _get_quality_scores(self, asset), which read the quality
        scores from the log file, and return the scores in a dictionary format.
        4) If necessary, override _remove_log(self, asset) if
        Executor._remove_log(self, asset) doesn't work for your purpose
        (sometimes the command-line executable could generate output log files
        in some different format, like multiple files).
    For an example, follow PsnrQualityRunner.

    b) Override the Executor._run_on_asset(self, asset) method to bypass the
    regular routine, but instead, in the method construct a FeatureAssembler
    (which calls a FeatureExtractor (or many) and assembles a list of features,
    followed by using a TrainTestModel (pre-trained somewhere else) to predict
    the final quality score. You must:
        1) Override TYPE and VERSION
        2) Override _run_on_asset(self, asset), which runs a FeatureAssembler,
        collect a feature vector, run TrainTestModel.predict() on it, and
        return a Result object (in this case, both Executor._run_on_asset(self,
        asset) and QualityRunner._read_result(self, asset) get bypassed.
        3) Override _remove_log(self, asset) by redirecting it to the
        FeatureAssembler.
        4) Override _remove_result(self, asset) by redirecting it to the
        FeatureAssembler.
    For an example, follow VmafQualityRunner.
    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _read_result(self, asset):
        result = {}
        result.update(self._get_quality_scores(asset))
        executor_id = self.executor_id
        return Result(asset, executor_id, result)

    @classmethod
    def get_scores_key(cls):
        return cls.TYPE + '_scores'

    @classmethod
    def get_score_key(cls):
        return cls.TYPE + '_score'


class PsnrQualityRunner(QualityRunner):

    TYPE = 'PSNR'
    VERSION = '1.0'

    def _generate_result(self, asset):
        # routine to call the command-line executable and generate quality
        # scores in the log file.

        quality_width, quality_height = asset.quality_width_height
        log_file_path = self._get_log_file_path(asset)

        yuv_type = self._get_workfile_yuv_type(asset)
        ref_path = asset.ref_workfile_path
        dis_path = asset.dis_workfile_path
        w = quality_width
        h = quality_height
        logger = self.logger

        ExternalProgramCaller.call_psnr(yuv_type, ref_path, dis_path, w, h, log_file_path, logger)

    def _get_quality_scores(self, asset):
        # routine to read the quality scores from the log file, and return
        # the scores in a dictionary format.

        log_file_path = self._get_log_file_path(asset)

        psnr_scores = []
        counter = 0
        with open(log_file_path, 'rt') as log_file:
            for line in log_file.readlines():
                mo = re.match(r"psnr: ([0-9]+) ([0-9.-]+)", line)
                if mo:
                    cur_idx = int(mo.group(1))
                    assert cur_idx == counter
                    psnr_scores.append(float(mo.group(2)))
                    counter += 1

        assert len(psnr_scores) != 0

        scores_key = self.get_scores_key()
        quality_result = {
            scores_key:psnr_scores
        }
        return quality_result


class VmafLegacyQualityRunner(QualityRunner):

    TYPE = 'VMAF_legacy'

    # VERSION = '1.1'
    VERSION = 'F' + VmafFeatureExtractor.VERSION + '-1.1'

    FEATURE_ASSEMBLER_DICT = {'VMAF_feature': 'all'}

    FEATURE_RESCALE_DICT = {'VMAF_feature_vif_scores': (0.0, 1.0),
                            'VMAF_feature_adm_scores': (0.4, 1.0),
                            'VMAF_feature_ansnr_scores': (10.0, 50.0),
                            'VMAF_feature_motion_scores': (0.0, 20.0)}

    SVM_MODEL_FILE = VmafConfig.model_path("other_models", "model_V8a.model")

    # model_v8a.model is trained with customized feature order:
    SVM_MODEL_ORDERED_SCORES_KEYS = ['VMAF_feature_vif_scores',
                                     'VMAF_feature_adm_scores',
                                     'VMAF_feature_ansnr_scores',
                                     'VMAF_feature_motion_scores']

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_vmaf_feature_assembler_instance(self, asset):
        vmaf_fassembler = FeatureAssembler(
            feature_dict=self.FEATURE_ASSEMBLER_DICT,
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=None,
            optional_dict2=None,
            parallelize=False, # parallelization already in a higher level
        )
        return vmaf_fassembler

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset), which runs a
        # FeatureAssembler, collect a feature vector, run
        # TrainTestModel.predict() on it, and return a Result object
        # (in this case, both Executor._run_on_asset(self, asset) and
        # QualityRunner._read_result(self, asset) get bypassed.

        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.run()
        feature_result = vmaf_fassembler.results[0]

        # =====================================================================

        try:
            svmutil
        except NameError:
            from vmaf import svmutil

        # SVR predict
        model = svmutil.svm_load_model(self.SVM_MODEL_FILE)

        ordered_scaled_scores_list = []
        for scores_key in self.SVM_MODEL_ORDERED_SCORES_KEYS:
            scaled_scores = self._rescale(feature_result[scores_key],
                                          self.FEATURE_RESCALE_DICT[scores_key])
            ordered_scaled_scores_list.append(scaled_scores)

        scores = []
        for score_vector in zip(*ordered_scaled_scores_list):
            vif, adm, ansnr, motion = score_vector
            xs = [[vif, adm, ansnr, motion]]
            score = svmutil.svm_predict([0], xs, model)[0][0]
            score = self._post_correction(motion, score)
            scores.append(score)

        result_dict = {}
        # add all feature result
        result_dict.update(feature_result.result_dict)
        # add quality score
        result_dict[self.get_scores_key()] = scores

        return Result(asset, self.executor_id, result_dict)

    def _post_correction(self, motion, score):
        # post-SVM correction
        if motion > 12.0:
            val = motion
            if val > 20.0:
                val = 20
            score *= ((val - 12) * 0.015 + 1)
        if score > 100.0:
            score = 100.0
        elif score < 0.0:
            score = 0.0
        return score

    @classmethod
    def _rescale(cls, vals, lower_upper_bound):
        lower_bound, upper_bound = lower_upper_bound
        vals = np.double(vals)
        vals = np.clip(vals, lower_bound, upper_bound)
        vals = (vals - lower_bound) / (upper_bound - lower_bound)
        return vals

    # override
    def _remove_result(self, asset):
        # Override Executor._remove_result by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()


class VmafQualityRunner(QualityRunner):

    TYPE = 'VMAF'

    # VERSION = '0.1' # using model nflxall_vmafv1.pkl, VmafFeatureExtractor VERSION 0.1
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv1.pkl") # trained with resource/param/vmaf_v1.py on private/resource/dataset/NFLX_dataset.py (30 subjects)

    # VERSION = '0.2' # using model nflxall_vmafv2.pkl, VmafFeatureExtractor VERSION 0.2.1
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv2.pkl") # trained with resource/param/vmaf_v2.py on private/resource/dataset/NFLX_dataset.py (30 subjects)

    # VERSION = '0.3' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (30 subjects)

    # VERSION = '0.3.1' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1, NFLX_dataset with 26 subjects (last 4 outliers removed)
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3a.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (26 subjects)

    # VERSION = '0.3.2'  # using model nflxall_vmafv4.pkl, VmafFeatureExtractor VERSION 0.2.2, NFLX_dataset with 26 subjects (last 4 outliers removed)
    # ALGO_VERSION = 0
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl")  # trained with resource/param/vmaf_v4.py on private/resource/dataset/NFLX_dataset.py (26 subjects)

    # using model vmaf_v0.6.1.pkl. VmafFeatureExtractor VERSION 0.2.4b.
    VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1'
    ALGO_VERSION = 2

    # trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17
    DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl")

    DEFAULT_FEATURE_DICT = {'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']} # for backward-compatible with older model only

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_vmaf_feature_assembler_instance(self, asset):

        # load TrainTestModel only to retrieve its 'feature_dict' extra info
        feature_dict = self._load_model(asset).get_appended_info('feature_dict')
        if feature_dict is None:
            feature_dict = self.DEFAULT_FEATURE_DICT

        vmaf_fassembler = FeatureAssembler(
            feature_dict=feature_dict,
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=None,
            optional_dict2=None,
            parallelize=False, # parallelization already in a higher level
        )
        return vmaf_fassembler

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset), which runs a
        # FeatureAssembler, collect a feature vector, run
        # TrainTestModel.predict() on it, and return a Result object
        # (in this case, both Executor._run_on_asset(self, asset) and
        # QualityRunner._read_result(self, asset) get bypassed.
        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.run()
        feature_result = vmaf_fassembler.results[0]
        model = self._load_model(asset)
        xs = model.get_per_unit_xs_from_a_result(feature_result)

        if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
            disable_clip_score = self.optional_dict['disable_clip_score']
        else:
            disable_clip_score = False

        if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
            enable_transform_score = self.optional_dict['enable_transform_score']
        else:
            enable_transform_score = False

        pred_result = self.predict_with_model(model, xs,
                                          disable_clip_score=disable_clip_score,
                                          enable_transform_score=enable_transform_score)
        result_dict = self._populate_result_dict(feature_result, pred_result)
        return Result(asset, self.executor_id, result_dict)

    def _populate_result_dict(self, feature_result, pred_result):
        result_dict = {}
        result_dict.update(feature_result.result_dict)  # add feature result
        result_dict[self.get_scores_key()] = pred_result['ys_pred']  # add quality score
        return result_dict

    @classmethod
    def predict_with_model(cls, model, xs, **kwargs):
        ys_pred = model.predict(xs)['ys_label_pred']
        do_transform_score = cls._do_transform_score(kwargs)
        if do_transform_score:
            ys_pred = cls.transform_score(model, ys_pred)
        else:
            pass
        if 'disable_clip_score' in kwargs and kwargs['disable_clip_score'] is True:
            pass
        else:
            ys_pred = cls.clip_score(model, ys_pred)
        return {'ys_pred': ys_pred}

    @staticmethod
    def _do_transform_score(kwargs):
        return 'enable_transform_score' in kwargs and kwargs['enable_transform_score'] is True

    @staticmethod
    def set_transform_score(model, score_transform):
        model.append_info('score_transform', score_transform)

    @staticmethod
    def set_clip_score(model, score_clip):
        model.append_info('score_clip', score_clip)

    @staticmethod
    def transform_score(model, ys_pred):
        """
        Do post processing: transform final quality score e.g. via polynomial
        {'p0': 1, 'p1': 1, 'p2': 0.5} means transform through 1 + x + 0.5 * x^2.
        For now, only support polynomail up to 2nd-order.
        """
        transform_dict = model.get_appended_info('score_transform')

        if transform_dict is None:
            return ys_pred

        y_in = ys_pred
        y_out = np.zeros(ys_pred.shape)

        # quadratic transform
        if 'p0' in transform_dict:
            y_out += transform_dict['p0']
        if 'p1' in transform_dict:
            y_out += transform_dict['p1'] * y_in
        if 'p2' in transform_dict:
            y_out += transform_dict['p2'] * y_in * y_in

        # rectification
        if 'out_lte_in' in transform_dict and transform_dict['out_lte_in'] == 'true':
            # output must be less than or equal to input
            y_out = np.minimum(y_out, y_in)
        if 'out_gte_in' in transform_dict and transform_dict['out_gte_in'] == 'true':
            # output must be greater than or equal to input
            y_out = np.maximum(y_out, y_in)

        return y_out

    @staticmethod
    def clip_score(model, ys_pred):
        """
        Do post processing: clip final quality score within e.g. [0, 100]
        """
        score_clip = model.get_appended_info('score_clip')
        if score_clip is not None:
            lb, ub = score_clip
            ys_pred = np.clip(ys_pred, lb, ub)

        return ys_pred

    def _load_model(self, asset):
        if self.optional_dict is not None \
                and 'model_filepath' in self.optional_dict \
                and self.optional_dict['model_filepath'] is not None:
            model_filepath = self.optional_dict['model_filepath']
        else:
            model_filepath = self.DEFAULT_MODEL_FILEPATH
        train_test_model_class = self.get_train_test_model_class()
        try:
            model = train_test_model_class.from_file(model_filepath, self.logger)
        except AssertionError as e:
            raise AssertionError("File {filepath} may not be a valid model file for class {cls}: {e}".
                                 format(filepath=model_filepath, cls=train_test_model_class.__name__, e=e))
        return model

    def get_train_test_model_class(self):
        return LibsvmNusvrTrainTestModel

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()


class EnsembleVmafQualityRunner(VmafQualityRunner):

    TYPE = 'EnsembleVMAF'

    VERSION = '{}-Ensemble'.format(VmafQualityRunner.VERSION)

    DEFAULT_MODEL_FILEPATH = [VmafConfig.model_path("vmaf_v0.6.1.pkl"), VmafConfig.model_path("vmaf_v0.6.1.pkl")]

    # this now needs to become a list
    DEFAULT_FEATURE_DICT = [{'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']}, {'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']}]

    def _populate_result_dict(self, feature_result, pred_result, result_dict):
        result_dict.update(feature_result.result_dict)  # add feature result
        result_dict[self.get_scores_key()] = pred_result['ys_pred']  # add quality score
        return result_dict

    def _get_ensemblevmaf_feature_assembler_instance(self, asset):

        # load TrainTestModel only to retrieve its 'feature_dict' extra info
        ensem_models = self._load_model(asset)
        ensemblevmaf_fassemblers = []

        for model_ind, model_now in enumerate(ensem_models):

            feature_dict = model_now.get_appended_info('feature_dict')
            if feature_dict is None:
                feature_dict = self.DEFAULT_FEATURE_DICT[model_ind]

            ensemblevmaf_fassembler = FeatureAssembler(
                feature_dict=feature_dict,
                feature_option_dict=None,
                assets=[asset],
                logger=self.logger,
                fifo_mode=self.fifo_mode,
                delete_workdir=self.delete_workdir,
                result_store=self.result_store,
                optional_dict=None,
                optional_dict2=None,
                parallelize=False,  # parallelization already in a higher level
            )

            ensemblevmaf_fassemblers.append(ensemblevmaf_fassembler)

        return ensemblevmaf_fassemblers

    def _get_Nframes(self, pred_result):

        return len(pred_result['ys_pred'])

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset), which runs a
        # FeatureAssembler, collect a feature vector, run
        # TrainTestModel.predict() on it, and return a Result object
        # (in this case, both Executor._run_on_asset(self, asset) and
        # QualityRunner._read_result(self, asset) get bypassed.
        ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance(asset)

        # each model is associated with a Feature Assembler
        Nmodels = len(ensemblevmaf_fassemblers)
        pred_result_ensem_models = []
        result_dict = {}

        for model_ind in range(Nmodels):

            evmaf_fassembler = ensemblevmaf_fassemblers[model_ind]
            evmaf_fassembler.run()
            feature_result = evmaf_fassembler.results[0]
            model = self._load_model(asset)[model_ind]
            xs = model.get_per_unit_xs_from_a_result(feature_result)

            if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
                disable_clip_score = self.optional_dict['disable_clip_score']
            else:
                disable_clip_score = False

            if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
                enable_transform_score = self.optional_dict['enable_transform_score']
            else:
                enable_transform_score = False

            pred_result = self.predict_with_model(model, xs,
                                              disable_clip_score=disable_clip_score,
                                              enable_transform_score=enable_transform_score)
            result_dict = self._populate_result_dict(feature_result, pred_result, result_dict)
            pred_result_ensem_models.append(pred_result)

        assert Nmodels > 0

        Nframes = self._get_Nframes(pred_result)

        all_model_scores = np.zeros((Nmodels, Nframes))
        all_model_score_names = self.ensemblevmaf_get_scores_key(Nmodels)
        for model_ind in range(Nmodels):
            result_dict[all_model_score_names[model_ind]] = pred_result_ensem_models[model_ind]['ys_pred']  # add quality score
            all_model_scores[model_ind, :] = pred_result_ensem_models[model_ind]['ys_pred']

        # perform prediction averaging (simple average for now)
        pred_result_ensem_models_aggregate = np.mean(all_model_scores, axis=0)

        # write results
        result_dict[self.get_scores_key()] = pred_result_ensem_models_aggregate

        return Result(asset, self.executor_id, result_dict)

    def ensemblevmaf_get_scores_key(self, Nmodels):
        scores_name_list = []
        for model_ind in range(Nmodels):
            scores_name_list.append(self.TYPE + '_model_' + str(model_ind) + '_scores')
        return scores_name_list

    def _load_model(self, asset):
        if self.optional_dict is not None \
                and 'model_filepath' in self.optional_dict \
                and self.optional_dict['model_filepath'] is not None:
            model_filepath = self.optional_dict['model_filepath']
        else:
            model_filepath = self.DEFAULT_MODEL_FILEPATH

        model = []
        for model_filepath_part in model_filepath:
            model.append(TrainTestModel.from_file(model_filepath_part, self.logger))
        return model

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance(asset)
        for ensemblevmaf_fassembler in ensemblevmaf_fassemblers:
            ensemblevmaf_fassembler.remove_results()


class VmafPhoneQualityRunner(VmafQualityRunner):

    TYPE = 'VMAF_Phone'

    VERSION = '{}-phone'.format(VmafQualityRunner.VERSION)

    def _assert_args(self):
        super(VmafPhoneQualityRunner, self)._assert_args()

        if self.optional_dict is not None:
            assert 'enable_transform_score' not in self.optional_dict, \
                'Cannot specify enable_transform_score option in {cls}.'.format(cls=self.__class__.__name__)

    @staticmethod
    def _do_transform_score(kwargs):
        return True


class VmafossExecQualityRunner(QualityRunner):

    TYPE = 'VMAFOSSEXEC'

    # VERSION = '0.3'
    # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3.pkl.model")

    # VERSION = '0.3.1'
    # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3a.pkl.model")

    # VERSION = '0.3.2'
    # ALGO_VERSION = 0
    # # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv4.pkl.model")
    # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl")

    VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1'
    ALGO_VERSION = 2

    # trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17
    DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl")

    FEATURES = ['adm2', 'adm_scale0', 'adm_scale1', 'adm_scale2', 'adm_scale3',
                'motion', 'vif_scale0', 'vif_scale1', 'vif_scale2',
                'vif_scale3', 'vif', 'psnr', 'ssim', 'ms_ssim', 'motion2',
                'bagging', 'stddev', 'ci95_low', 'ci95_high']

    @classmethod
    def get_feature_scores_key(cls, atom_feature):
        return "{type}_{atom_feature}_scores".format(
            type=cls.TYPE, atom_feature=atom_feature)

    def _generate_result(self, asset):
        # routine to call the command-line executable and generate quality
        # scores in the log file.

        log_file_path = self._get_log_file_path(asset)

        if self.optional_dict is not None \
                and 'model_filepath' in self.optional_dict \
                and self.optional_dict['model_filepath'] is not None:
            model_filepath = self.optional_dict['model_filepath']
        else:
            model_filepath = self.DEFAULT_MODEL_FILEPATH

        if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
            disable_clip_score = self.optional_dict['disable_clip_score']
        else:
            disable_clip_score = False

        if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
            enable_transform_score = self.optional_dict['enable_transform_score']
        else:
            enable_transform_score = False

        if self.optional_dict is not None and 'phone_model' in self.optional_dict:
            phone_model = self.optional_dict['phone_model']
        else:
            phone_model = False

        if self.optional_dict is not None and 'disable_avx' in self.optional_dict:
            disable_avx = self.optional_dict['disable_avx']
        else:
            disable_avx = False

        if self.optional_dict is not None and 'thread' in self.optional_dict:
            n_thread = self.optional_dict['thread']
        else:
            n_thread = 0

        if self.optional_dict is not None and 'subsample' in self.optional_dict:
            n_subsample = self.optional_dict['subsample']
        else:
            n_subsample = 1

        if self.optional_dict is not None and 'psnr' in self.optional_dict:
            psnr = self.optional_dict['psnr']
        else:
            psnr = True

        if self.optional_dict is not None and 'ssim' in self.optional_dict:
            ssim = self.optional_dict['ssim']
        else:
            ssim = True

        if self.optional_dict is not None and 'ms_ssim' in self.optional_dict:
            ms_ssim = self.optional_dict['ms_ssim']
        else:
            ms_ssim = True

        if self.optional_dict is not None and 'ci' in self.optional_dict:
            ci = self.optional_dict['ci']
        else:
            ci = False

        quality_width, quality_height = asset.quality_width_height

        fmt=self._get_workfile_yuv_type(asset)
        w=quality_width
        h=quality_height
        ref_path=asset.ref_workfile_path
        dis_path=asset.dis_workfile_path
        model=model_filepath
        exe=self._get_exec()
        logger = self.logger

        ExternalProgramCaller.call_vmafossexec(fmt, w, h, ref_path, dis_path, model, log_file_path,
                                               disable_clip_score, enable_transform_score,
                                               phone_model, disable_avx, n_thread, n_subsample,
                                               psnr, ssim, ms_ssim, ci, exe, logger)

    def _get_exec(self):
        return None # signaling default

    def _get_quality_scores(self, asset):
        # routine to read the quality scores from the log file, and return
        # the scores in a dictionary format.
        log_file_path = self._get_log_file_path(asset)
        tree = ElementTree.parse(log_file_path)
        root = tree.getroot()
        scores = []

        # check if vmafossexec returned additional info about the bootstrapped models
        # bootstrap_model_list_str is a comma-separated string of model names
        if 'bootstrap_model_list_str' in root.findall('params')[0].attrib:
            bootstrap_model_list = []
            vmaf_params = root.findall('params')[0].attrib
            bootstrap_model_list_str = vmaf_params['bootstrap_model_list_str']
            bootstrap_model_list = bootstrap_model_list_str.split(',')
            # augment the feature set with bootstrap models
            self.FEATURES += bootstrap_model_list

        feature_scores = [[] for _ in self.FEATURES]

        for frame in root.findall('frames/frame'):
            scores.append(float(frame.attrib['vmaf']))
            for i_feature, feature in enumerate(self.FEATURES):
                try:
                    feature_scores[i_feature].append(float(frame.attrib[feature]))
                except KeyError:
                    pass # some features may be missing
        assert len(scores) != 0
        quality_result = {
            self.get_scores_key(): scores,
        }
        for i_feature, feature in enumerate(self.FEATURES):
            if len(feature_scores[i_feature]) != 0:
                quality_result[self.get_feature_scores_key(feature)] = feature_scores[i_feature]
        return quality_result


class SsimQualityRunner(QualityRunner):

    TYPE = 'SSIM'
    VERSION = '1.0'

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_feature_assembler_instance(self, asset):

        feature_dict = {SsimFeatureExtractor.TYPE: SsimFeatureExtractor.ATOM_FEATURES}

        feature_assembler = FeatureAssembler(
            feature_dict=feature_dict,
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=None,
            optional_dict2=None,
            parallelize=False, # parallelization already in a higher level
        )
        return feature_assembler

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset)
        vmaf_fassembler = self._get_feature_assembler_instance(asset)
        vmaf_fassembler.run()
        feature_result = vmaf_fassembler.results[0]
        result_dict = {}
        result_dict.update(feature_result.result_dict.copy()) # add feature result
        result_dict[self.get_scores_key()] = feature_result.result_dict[
            SsimFeatureExtractor.get_scores_key('ssim')] # add ssim score
        del result_dict[SsimFeatureExtractor.get_scores_key('ssim')] # delete redundant
        return Result(asset, self.executor_id, result_dict)

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()


class MsSsimQualityRunner(QualityRunner):

    TYPE = 'MS_SSIM'
    VERSION = '1.0'

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_feature_assembler_instance(self, asset):

        feature_dict = {MsSsimFeatureExtractor.TYPE: MsSsimFeatureExtractor.ATOM_FEATURES}

        feature_assembler = FeatureAssembler(
            feature_dict=feature_dict,
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=None,
            optional_dict2=None,
            parallelize=False, # parallelization already in a higher level
        )
        return feature_assembler

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset)
        vmaf_fassembler = self._get_feature_assembler_instance(asset)
        vmaf_fassembler.run()
        feature_result = vmaf_fassembler.results[0]
        result_dict = {}
        result_dict.update(feature_result.result_dict.copy()) # add feature result
        result_dict[self.get_scores_key()] = feature_result.result_dict[
            MsSsimFeatureExtractor.get_scores_key('ms_ssim')] # add ssim score
        del result_dict[MsSsimFeatureExtractor.get_scores_key('ms_ssim')] # delete redundant
        return Result(asset, self.executor_id, result_dict)

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()


class VmafSingleFeatureQualityRunner(QualityRunner):

    __metaclass__ = ABCMeta

    VERSION = 'F{}-0'.format(VmafFeatureExtractor.VERSION)

    @property
    @abstractmethod
    def FEATURE_NAME(self):
        raise NotImplementedError

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_vmaf_feature_assembler_instance(self, asset):

        vmaf_fassembler = FeatureAssembler(
            feature_dict={'VMAF_feature': [self.FEATURE_NAME]},
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=None,
            optional_dict2=None,
            parallelize=False, # parallelization already in a higher level
        )
        return vmaf_fassembler

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset)
        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.run()
        feature_result = vmaf_fassembler.results[0]
        result_dict = {
            self.get_scores_key(): feature_result[VmafFeatureExtractor.get_scores_key(self.FEATURE_NAME)]
        }

        return Result(asset, self.executor_id, result_dict)

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()


class VifQualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF'
    FEATURE_NAME = 'vif'


class Vif2QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF2'
    FEATURE_NAME = 'vif2'


class Adm2QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'ADM2'
    # TYPE = 'DLM'
    FEATURE_NAME = 'adm2'


class VifScale0QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF_SCALE0'
    FEATURE_NAME = 'vif_scale0'


class VifScale1QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF_SCALE1'
    FEATURE_NAME = 'vif_scale1'


class VifScale2QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF_SCALE2'
    FEATURE_NAME = 'vif_scale2'


class VifScale3QualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'VIF_SCALE3'
    FEATURE_NAME = 'vif_scale3'


class MotionQualityRunner(VmafSingleFeatureQualityRunner):
    TYPE = 'MOTION'
    # TYPE = 'TI'
    FEATURE_NAME = 'motion'


class BootstrapVmafQualityRunner(VmafQualityRunner):

    TYPE = "BOOTSTRAP_VMAF"
    VERSION = VmafQualityRunner.VERSION + '-' + 'M' + BootstrapLibsvmNusvrTrainTestModel.VERSION
    ALGO_VERSION = None

    # "vmaf_b_v0.6.3": plain bootstrapping, "vmaf_rb_v0.6.3": residue bootstrapping
    DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_b_v0.6.3", "vmaf_b_v0.6.3.pkl")

    def _populate_result_dict(self, feature_result, pred_result):
        result_dict = {}
        result_dict.update(feature_result.result_dict)  # add feature result
        result_dict[self.get_scores_key()] = pred_result['ys_pred']  # add quality score
        result_dict[self.get_all_models_scores_key()] = pred_result['ys_pred_all_models']  # add quality score from all models
        result_dict[self.get_bagging_scores_key()] = pred_result['ys_pred_bagging']  # add bagging quality score
        result_dict[self.get_stddev_scores_key()] = pred_result['ys_pred_stddev']  # add stddev of bootstrapped quality score
        result_dict[self.get_ci95_low_scores_key()] = pred_result['ys_pred_ci95_low']  # add ci95 of bootstrapped quality score
        result_dict[self.get_ci95_high_scores_key()] = pred_result['ys_pred_ci95_high']  # add ci95 of bootstrapped quality score
        return result_dict

    @classmethod
    def predict_with_model(cls, model, xs, **kwargs):
        DELTA = 1e-2
        result = model.predict(xs)
        ys_pred_all_models = result['ys_label_pred_all_models']
        ys_pred = result['ys_label_pred']
        ys_pred_bagging = result['ys_label_pred_bagging']
        ys_pred_stddev = result['ys_label_pred_stddev']
        ys_pred_ci95_low = result['ys_label_pred_ci95_low']
        ys_pred_ci95_high = result['ys_label_pred_ci95_high']
        ys_pred_plus = ys_pred_bagging + DELTA
        ys_pred_minus = ys_pred_bagging - DELTA

        do_transform_score = cls._do_transform_score(kwargs)
        if do_transform_score:
            ys_pred_all_models = np.array([cls.transform_score(model, ys_pred_some_model) for ys_pred_some_model in ys_pred_all_models])
            ys_pred = cls.transform_score(model, ys_pred)
            ys_pred_bagging = cls.transform_score(model, ys_pred_bagging)
            ys_pred_plus = cls.transform_score(model, ys_pred_plus)
            ys_pred_minus = cls.transform_score(model, ys_pred_minus)
            ys_pred_ci95_low = cls.transform_score(model, ys_pred_ci95_low)
            ys_pred_ci95_high = cls.transform_score(model, ys_pred_ci95_high)
        else:
            pass

        if 'disable_clip_score' in kwargs and kwargs['disable_clip_score'] is True:
            pass
        else:
            ys_pred_all_models = np.array([cls.clip_score(model, ys_pred_some_model) for ys_pred_some_model in ys_pred_all_models])
            ys_pred = cls.clip_score(model, ys_pred)
            ys_pred_bagging = cls.clip_score(model, ys_pred_bagging)
            ys_pred_plus = cls.clip_score(model, ys_pred_plus)
            ys_pred_minus = cls.clip_score(model, ys_pred_minus)
            ys_pred_ci95_low = cls.clip_score(model, ys_pred_ci95_low)
            ys_pred_ci95_high = cls.clip_score(model, ys_pred_ci95_high)

        # stddev score transform is applied after transform, clip, or both, or neither
        slope = ((ys_pred_plus - ys_pred_minus) / (2.0 * DELTA))
        ys_pred_stddev = ys_pred_stddev * slope

        return {'ys_pred_all_models': ys_pred_all_models,
                'ys_pred': ys_pred,
                'ys_pred_bagging': ys_pred_bagging,
                'ys_pred_stddev': ys_pred_stddev,
                'ys_pred_ci95_low': ys_pred_ci95_low,
                'ys_pred_ci95_high': ys_pred_ci95_high,
                }

    def get_train_test_model_class(self):
        # overide VmafQualityRunner.get_train_test_model_class
        return BootstrapLibsvmNusvrTrainTestModel

    @classmethod
    def get_all_models_scores_key(cls):
        return cls.TYPE + '_all_models_scores'

    @classmethod
    def get_all_models_score_key(cls):
        return cls.TYPE + '_all_models_score'

    @classmethod
    def get_bagging_scores_key(cls):
        return cls.TYPE + '_bagging_scores'

    @classmethod
    def get_bagging_score_key(cls):
        return cls.TYPE + '_bagging_score'

    @classmethod
    def get_stddev_scores_key(cls):
        return cls.TYPE + '_stddev_scores'

    @classmethod
    def get_stddev_score_key(cls):
        return cls.TYPE + '_stddev_score'

    @classmethod
    def get_ci95_low_scores_key(cls):
        return cls.TYPE + '_ci95_low_scores'

    @classmethod
    def get_ci95_low_score_key(cls):
        return cls.TYPE + '_ci95_low_score'

    @classmethod
    def get_ci95_high_scores_key(cls):
        return cls.TYPE + '_ci95_high_scores'

    @classmethod
    def get_ci95_high_score_key(cls):
        return cls.TYPE + '_ci95_high_score'


class BaggingVmafQualityRunner(BootstrapVmafQualityRunner):

    TYPE = "BAGGING_VMAF"
    VERSION = VmafQualityRunner.VERSION + '-' + BootstrapLibsvmNusvrTrainTestModel.VERSION

    def _populate_result_dict(self, feature_result, pred_result):
        result_dict = super(BaggingVmafQualityRunner, self)._populate_result_dict(feature_result, pred_result)

        # override score with bagging score
        result_dict[self.get_scores_key()] = pred_result['ys_pred_bagging']

        return result_dict


class NiqeQualityRunner(QualityRunner):

    TYPE = 'NIQE'

    VERSION = '0.1'
    DEFAULT_MODEL_FILEPATH = VmafConfig.model_path('other_models', 'niqe_v0.1.pkl')

    DEFAULT_FEATURE_DICT = {'NIQE_noref_feature': 'all'}

    def _get_quality_scores(self, asset):
        raise NotImplementedError

    def _generate_result(self, asset):
        raise NotImplementedError

    def _get_niqe_feature_assembler_instance(self, asset):

        # load TrainTestModel only to retrieve its 'feature_dict' extra info
        model = self._load_model(asset)

        # need this so that FeatureAssembler can find NiqeNorefFeatureExtractor:
        from vmaf.core.noref_feature_extractor import NiqeNorefFeatureExtractor

        feature_dict = model.get_appended_info('feature_dict')
        if feature_dict is None:
            feature_dict = self.DEFAULT_FEATURE_DICT

        feature_optional_dict = model.get_appended_info('feature_optional_dict')

        vmaf_fassembler = FeatureAssembler(
            feature_dict=feature_dict,
            feature_option_dict=None,
            assets=[asset],
            logger=self.logger,
            fifo_mode=self.fifo_mode,
            delete_workdir=self.delete_workdir,
            result_store=self.result_store,
            optional_dict=feature_optional_dict,
            optional_dict2=None,
            parallelize=False,
        )

        return vmaf_fassembler

    def _load_model(self, asset):
        if self.optional_dict is not None \
                and 'model_filepath' in self.optional_dict \
                and self.optional_dict['model_filepath'] is not None:
            model_filepath = self.optional_dict['model_filepath']
        else:
            model_filepath = self.DEFAULT_MODEL_FILEPATH
        model = TrainTestModel.from_file(model_filepath, self.logger)
        return model

    def _run_on_asset(self, asset):
        # Override Executor._run_on_asset(self, asset), which runs a
        # FeatureAssembler, collect a feature vector, run
        # TrainTestModel.predict() on it, and return a Result object
        # (in this case, both Executor._run_on_asset(self, asset) and
        # QualityRunner._read_result(self, asset) get bypassed.

        niqe_fassembler = self._get_niqe_feature_assembler_instance(asset)
        niqe_fassembler.run()
        feature_result = niqe_fassembler.results[0]

        # xs = NiqeTrainTestModel.get_perframe_xs_from_result(feature_result)
        xs = NiqeTrainTestModel.get_xs_from_results([feature_result])

        model = self._load_model(asset)

        ys_pred = model.predict(xs)['ys_label_pred']

        result_dict = {}
        # add all feature result
        result_dict.update(feature_result.result_dict)
        # add quality score
        result_dict[self.get_scores_key()] = ys_pred

        return Result(asset, self.executor_id, result_dict)

    def _remove_result(self, asset):
        # Override Executor._remove_result(self, asset) by redirecting it to the
        # FeatureAssembler.

        vmaf_fassembler = self._get_niqe_feature_assembler_instance(asset)
        vmaf_fassembler.remove_results()